# -*- coding: utf-8 -*-
from scrapy.selector import Selector
import scrapy
from scrapy.loader import ItemLoader, Identity
from spiders.items import JinSeItem


class JinSeSpider(scrapy.Spider):
    name = "jinse"
    allowed_domains = ["jinse.com"]
    base_url = 'https://www.jinse.com/'
    start_urls = ["https://www.jinse.com/"]
    # 自定义配置 pipeline
    custom_settings = {
        'ITEM_PIPELINES': {
            # 'spiders.pipelines.TestPipeline.TestPipeline': 1,
            'spiders.pipelines.OkexPipeline.OkexPipeline': 1,
        },
        # 减慢爬取速度为1s
        'DOWNLOAD_DELAY': 1,
        # 指定redis数据库的连接参数
        'REDIS_HOST': '127.0.0.1',
        # 端口
        'REDIS_PORT': 6379,
        # 指定 redis链接密码，和使用哪一个数据库
        'REDIS_PARAMS': {
            'password': 'Aa123456',
            'db': 1,
        },
    }

    def parse(self, response):
        sel = Selector(response)
        link = sel.xpath("//div[@class='article-main']/div/ol/a/@href").extract()
        for x in link:
            request = scrapy.Request(x, callback=self.parse_item)
            print(request)
            yield request

    def parse_item(self, response):
        item = ItemLoader(item=JinSeItem(), response=response)
        # 修改图片并下载
        images_url = response.selector.xpath('content_image_urls', "//div[@class='js-article']/p//img/@src").extract()
        for x in images_url:
            self.images_edit(x)

        item.add_xpath('title', "//div[@class='js-article']/div/h2/text()")
        item.add_xpath('author', "//div[@class='js-article']/div[@class='article-info']/a/text()")
        item.add_xpath('click', "//div[@class='js-article']/div[@class='article-info']/ul/span/text()")
        item.add_xpath('create_time', "//div[@class='js-article']//div[@class='time']/text()")
        item.add_xpath('author', "//div[@class='js-article']/div[@class='article-info']/a/text()")
        item.add_xpath('content', "//div[@class='js-article']/p")
        return item.load_item()

    # 修改地图地址并保存
    def images_edit(self, image_url):
        res = image_url.replace("https://img.jinse.com", '')
        print(res)
        return res
